﻿using System;
using System.Net;
using System.IO;
using System.Text;
using System.Collections.Generic;
using System.Windows.Forms;
using mshtml;
using System.Xml;
using System.Linq;
using System.Xml.Serialization;
using System.Runtime.InteropServices;
using System.Runtime.Serialization.Formatters.Binary;
using System.Runtime.InteropServices.ComTypes;
using System.Text.RegularExpressions;
using System.IO.Compression;


namespace hkjc
{
    public static class WebFunctions
    {
        public static string OpenUrl(string Url)
        {
            string ret = string.Empty;
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "GET";
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                string charSet = response.CharacterSet;
                Encoding encoding;
                if (string.IsNullOrEmpty(charSet))
                    encoding = Encoding.Default;
                else
                    encoding = Encoding.GetEncoding(charSet);

                StreamReader sr = new StreamReader(response.GetResponseStream(), encoding);
                ret = sr.ReadToEnd();                
                sr.Close();
                response.Close();
            }
            catch (Exception e)
            {
                ret = string.Empty;
                Program.LastErrorMsg = "[ERROR]Get html error:"+e.Message;
            }
            //ret = charSet;
            
            return ret;
        }
        public static void ParseUrls(string strPattern,ref List<string> Urls)
        {
            strPattern = strPattern.Replace(" ", "");
            //check the number of parameters,limited to 5
            string[] ps = strPattern.Split(new string[]{"["},System.StringSplitOptions.RemoveEmptyEntries);
            if (ps.GetUpperBound(0)>5)
            {
                MessageBox.Show("Max Support 5 Parameters");
                return;
            }

            for (int i = 1; i <= ps.GetUpperBound(0); i++)
            {
                // no ']' enclose the parameter,error
                int j = ps[i].IndexOf("]");
                if (j <= 0)
                {
                    MessageBox.Show("URL Parameters Error:" + ps[i]);
                    return;
                }
                string S = ps[i].Substring(0, j);
                strPattern = strPattern.Replace("[" + S + "]", "[P" + (i+1) + "]");
                ps[i] = "";
                // "-" stands for consecutive series, so change it to ";" pattern
                if (S.IndexOf("-") >= 0)
                {
                    string s1 = S.Split(new string[] { "-" }, System.StringSplitOptions.RemoveEmptyEntries)[0]; // start
                    string s2 = S.Split(new string[] { "-" }, System.StringSplitOptions.RemoveEmptyEntries)[1]; // end
                    int m = s1[s1.Length - 1]; // the last char
                    int n = s2[s2.Length - 1];
                    int k1, k2;
                    if (int.TryParse(s1, out k1) && int.TryParse(s2, out k2)) // it 's number
                        for (int k = k1; k <= k2; k++)
                            ps[i] = ps[i] + (k == k1 ? "" : ";") + k.ToString().PadLeft(s1.Length, '0');

                    else if ((m >= 65 && m <= 90 || m >= 97 && m <= 122) && (n >= 65 && n <= 90 || n >= 97 && n <= 122)) // it's a character
                        for (int k = m; k <= n; k++)
                            ps[i] = ps[i] + (k == m ? "" : ";") + s1.Substring(0, s1.Length - 1) + Convert.ToChar(k);  // note : only the last character take effects,e.g. aab-abc -> aab;aac
                }
                else
                    ps[i] = S;
            }

            //make parameter list to fit 5 items
            if (ps.GetUpperBound(0) < 4)
                Array.Resize(ref ps, 5);
            for (int i = 0; i < 5; i++)
                if (string.IsNullOrEmpty(ps[i]))
                    ps[i] = "";

            foreach (string p1 in ps[0].Split(new string[] { ";" }, System.StringSplitOptions.None) )
                foreach ( string p2 in ps[1].Split(new string[] { ";" }, System.StringSplitOptions.None))
                    foreach (string p3 in ps[2].Split(new string[] { ";" }, System.StringSplitOptions.None))
                        foreach (string p4 in ps[3].Split(new string[] { ";" }, System.StringSplitOptions.None))
                            foreach (string p5 in ps[4].Split(new string[] { ";" }, System.StringSplitOptions.None))
                            {
                                if (Urls.Count > 1000)
                                {
                                    MessageBox.Show("Number Of Urls Limited To 1000");
                                    return;
                                }
                                //do some log and wait for seconds
                                Urls.Add(strPattern.Replace("[P1]", p1).Replace("[P2]", p2).Replace("[P3]", p3).Replace("[P4]", p4).Replace("[P5]", p5).Replace(" ",""));
                            }

        }
        public static TResult RunXLScraperOnHtml(string txtHtml, ScraperData sd, ref List<string[]> rows)
        {
            TResult ret = new TResult();
            string s;

            if (txtHtml == string.Empty)
            {
                ret.ErrMsg = "[ERROR]The Html text is null,check the url if correct";
                ret.Success = false;
                return ret;
            }
            //rows.Clear();
            string[] rowdata = new string[30];
            int col = 0;

            try
            {
                string[] ps = txtHtml.Split(new string[] { sd.ScraperRows[0].MakerBefore }, StringSplitOptions.None);
                for (int i = 1; i < ps.Length; i++)
                {
                    s = ps[i];
                    col = 0;

                    for (int j = 0; j < sd.ScraperRows.Count; j++)
                    {
                        int pos = s.IndexOf(sd.ScraperRows[j].MakerAfter);
                        if (pos < 0 || s.Length == 0)
                            break;
                        //add item ; column #1      
                        //if (j == 0)
                        //    lvi.Text = s.Substring(0, pos);
                        //else
                        //    lvi.SubItems.Add(s.Substring(0, pos));

                        rowdata[col++] = s.Substring(0, pos);

                        //s = s.Substring(0, pos + sd.ScraperRows[j].MakerAfter.Length);  //previous maker after may be equal to next maker before

                        //add subitems of the item
                        while (j < sd.ScraperRows.Count - 1)
                        {
                            pos = s.IndexOf(sd.ScraperRows[j + 1].MakerBefore) + sd.ScraperRows[j + 1].MakerBefore.Length;
                            if (pos < 0)
                                j = j + 1;
                            else
                                break;
                        }
                        if (pos >= 0)
                            s = s.Substring(pos);
                        else
                            break;
                    }
                    //lv.Items.Add(lvi);
                    rows.Add(rowdata);
                    //ThreadSafe_ListViewAddItem(iListViewIndex, lvi);
                }
            }
            catch (Exception e)
            {
                ret.Success = false;
                ret.ErrMsg = "[ERROR] Scrape on html failed,exception=" + e.Message;
            }
            return ret;
        }
        public static TResult RunEasyScraperOnUrl(string url, EasyScraper es, ref List<string[]> rows)
        {
            TResult ret = new TResult();
           HTMLDocument Doc, Doc2;

            IHTMLElementCollection BodyElementCollection;
            try
            {

                Doc2 = new mshtml.HTMLDocument();

                IPersistStreamInit ipstream = (IPersistStreamInit)Doc2;
                ipstream.InitNew();

                Doc = (HTMLDocument)Doc2.createDocumentFromUrl(url, null);

                while (Doc.readyState != "complete")
                    Application.DoEvents();

                if (url.Contains("racing/Info/Meeting"))
                    ret.returnvalue = GetRaceTime(Doc.getElementsByTagName("tbody"));

                BodyElementCollection = Doc.getElementsByTagName("html");

                RunEasyScaper(BodyElementCollection, 0, es.iLevel + ";" + es.sParentTag + ";" + es.sChildrenTags, es.sValidFlags,ref rows);
            }
            catch (Exception e)
            {
                ret.Success = false;
                ret.ErrMsg = "[ERROR]Run scraper failed,url=" + url + " error:" + e.Message;                
            }
            Doc = null;
            Doc2 = null;
            BodyElementCollection = null;
            return ret;
        }

        public static string CleanInvalidXmlChars(string text)
        {
            // From xml spec valid chars:
            // #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
            // any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
            string re = @"[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-u10FFFF]";
            return Regex.Replace(text, re, "");
        }

        public static string GetXmlContent(string Url)
        {
            string ret = string.Empty;
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "GET";
                request.ContentType = "text/xml";
                request.Accept = "gzip, deflate";
                
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                while (response.StatusCode != HttpStatusCode.OK)
                    Application.DoEvents();

                string charSet = response.CharacterSet;
                Encoding encoding;
                if (string.IsNullOrEmpty(charSet))
                    encoding = Encoding.Default;
                else
                    encoding = Encoding.GetEncoding(charSet);
                

                XmlTextReader myXMLReader = null;   //Declare XMLReader

                //myXMLReader = new XmlTextReader(response.GetResponseStream());
                if (response.ContentEncoding == "gzip")
                {
                    GZipStream gs = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                    myXMLReader = new XmlTextReader(gs);
                }
                else
                    myXMLReader = new XmlTextReader(response.GetResponseStream());
                //myXMLReader.MoveToContent();
                while (myXMLReader.Read())
                {
                    if (myXMLReader.Name == "OUT") 
                    {
                        ret = myXMLReader.ReadString();
                        break;
                    }
                }

                //ret = myXMLReader.ReadContentAsString();
                myXMLReader.Close();
                response.Close();
            }
            catch (Exception e)
            {
                ret = string.Empty;
                Program.LastErrorMsg = "[ERROR]Get xml error:" + e.Message;
            }
            //ret = charSet;

            return ret;
        }

        public static string GetInvestmentStatSql(string Url,string sqlpattern,string racedate,string venue,string raceno,out string batchno)
        {
            string ret = string.Empty;
            Program.LastErrorMsg = string.Empty;

            //initialize pool data
            Dictionary<string, string> dictPool = new Dictionary<string, string> { };
            string pools = "win;pla;qin;qpl;dbl;tce;qtt;tri;f2f;d2t;t2t;sixup;tbl".ToUpper();
            string[] ps = pools.Split(new string[] { ";" }, StringSplitOptions.None);
            foreach (string poolname in ps)
                dictPool.Add(poolname, "0");

            //string batchno = string.Empty;

            //string sd = racedate.Substring(6) + "-" + racedate.Substring(4, 2) + "-" + racedate.Substring(0, 4);
            batchno = string.Empty;
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "GET";

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                string charSet = response.CharacterSet;
                Encoding encoding;
                if (string.IsNullOrEmpty(charSet))
                    encoding = Encoding.Default;
                else
                    encoding = Encoding.GetEncoding(charSet);

                XmlTextReader myXMLReader = null;   //Declare XMLReader

                myXMLReader = new XmlTextReader(response.GetResponseStream());
                //TextReader tr = File.OpenText("C:\\Users\\Administrator\\Desktop\\xxx.xml");
                //myXMLReader = new XmlTextReader(tr);
                myXMLReader.WhitespaceHandling = WhitespaceHandling.Significant;
                

                while (myXMLReader.Read())
                {
                    if (myXMLReader.Name == "INV") 
                    {
                        if (string.IsNullOrEmpty(batchno))
                        {
                            batchno = myXMLReader.GetAttribute("TIME").ToString();

                            if (DateTime.Now.ToString("yyyyMMdd") == racedate && string.Compare(batchno, "233000") > 0)
                                batchno = DateTime.Now.AddDays(-1).ToString("MMdd") + batchno;
                            else
                                batchno = DateTime.Now.ToString("MMdd") +  batchno;
                        }
                        
                        if(!string.IsNullOrEmpty(myXMLReader.GetAttribute("POOL")))
                            dictPool[myXMLReader.GetAttribute("POOL").ToString().Replace("-", "2").Replace("6UP", "SIXUP")] = myXMLReader.ReadString();
                    }
                }
                DateTime dt;
                if (DateTime.TryParseExact(batchno, "MMddHHmmss", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out dt))
                {
                    object[] args = new object[6 + ps.GetUpperBound(0) + 1];
                    args[0] = batchno;
                    args[1] = racedate;
                    args[2] = venue;
                    args[3] = raceno;
                    int k = 4;
                    foreach (string value in dictPool.Keys)
                        args[k++] = dictPool[value];
                    args[k++] = DateTime.Now.ToString("yyyyMMdd HH:mm:ss");
                    args[k++] = "";
                    ret = string.Format(sqlpattern, args);
                }
                else
                    ret = string.Empty;
                myXMLReader.Close();
                response.Close();
            }
            catch (Exception e)
            {
                ret = string.Empty;
                Program.LastErrorMsg = "[ERROR]Get GetInvestmentStatSql error:" + e.Message;
            }

            return ret;
        }

        public static string GetRaceTime(IHTMLElementCollection ec)
        {
            //<TBODY sizcache="2" sizset="26">
            //<TR sizcache="2" sizset="26">
            //<TD><SPAN class=bold>第 3 場&nbsp;-&nbsp;香港大學專業進修學院讓賽</SPAN> <BR>2014年6月15日,星期日, 沙田, 13:30 <BR>草地, "C" 賽道, 1200米<BR>獎金: $550,000.00, 評分:40-15, 第五班 </TD></TR></TBODY>
            string rt = string.Empty;
            
            foreach (IHTMLElement e in ec)
            {
                if (e.getAttribute("sizset") != null)                    
                if (e.getAttribute("sizset") == 23)
                {
                    foreach (IHTMLElement e2 in e.all)
                    {
                        if (e2.tagName == "TD")
                        {
                            string s = e2.innerHTML;
                            string[] ps = s.Split(new string[] { "<BR>" }, StringSplitOptions.None)[1].Split(new string[] { "," }, StringSplitOptions.None);
                            rt = ps[ps.GetUpperBound(0)].Trim();
                        }
                    }
                }
            }

            return rt;
        }
        public static TResult RunEasyScaper(IHTMLElementCollection xElementCollection,
            int iLevel,
            string sKey,
            string sValidFlags,    
            ref List<string[]> rows)
        {
            TResult ret = new TResult();
            string[] ps = new string[] { };
            string sInnerText, href, fieldvalue;
            int i, k;
            bool fCheck;

            //ListView lv = m_listviews[iListViewIndex];
            //Dictionary<string, int>.KeyCollection kc = dc.Keys;
            //string s = kc.ElementAt(index);

            ps = sKey.Split(new string[] { ";" }, StringSplitOptions.None);
            string[] vs = sValidFlags.Split(new string[] { ";" }, StringSplitOptions.None);
            string[] rowdata ;
            int col = 0;

            if (iLevel == int.Parse(ps[0]))
            {
                //retrieve children innertext and href
                foreach (IHTMLElement parentElement in xElementCollection)
                {
                    if (parentElement.tagName == ps[1])
                    {
                        i = 2;
                        fCheck = true;
                        foreach (IHTMLElement childrenElement in parentElement.children)
                        {
                            //check elements if satifies
                            if (i > ps.GetUpperBound(0))
                                //fCheck = true;  // why not false ?sometimes the length of elementcollection changes by ajax
                                break;
                            if (ps[i] == childrenElement.tagName)
                                i = i + 1;
                            else
                            {
                                fCheck = false;
                                break;
                            }
                        }
                        if (fCheck && i == ps.GetUpperBound(0) + 1)
                        {
                            k = 0;
                            col = 0;
                            ListViewItem lvi = new ListViewItem();
                            rowdata = new string[30];
                            foreach (IHTMLElement childrenElement in parentElement.children)
                            {
                                
                                fieldvalue = string.Empty;
                                sInnerText = GetInnerText(childrenElement);
                                href = string.Empty;
                                if (childrenElement.getAttribute("href") != null)
                                    href = childrenElement.getAttribute("href").ToString();

                                if (vs[k] == "1")
                                {
                                    if (!String.IsNullOrEmpty(sInnerText))
                                        fieldvalue = sInnerText;
                                    else if (!String.IsNullOrEmpty(href))
                                        fieldvalue = href;

                                    if (k <= vs.GetUpperBound(0))
                                        rowdata[col++] = fieldvalue;
                                    //if (String.IsNullOrEmpty(lvi.Text))
                                    //    lvi.Text = fieldvalue == string.Empty ? " " : fieldvalue;
                                    //else if (k <= vs.GetUpperBound(0))
                                    //    lvi.SubItems.Add(fieldvalue);
                                    else
                                        break;
                                }
                                k = k + 1;
                            }
                            //lv.Items.Add(lvi);                            
                            rows.Add(rowdata);
                            //ThreadSafe_ListViewAddItem(iListViewIndex, lvi);
                        }
                    }
                }
            }
            foreach (IHTMLElement parentElement in xElementCollection)
            {
                try
                {
                    if (parentElement.children.Length > 0)
                        ret = RunEasyScaper(parentElement.children, iLevel + 1, sKey, sValidFlags, ref rows);
                }
                catch (Exception e)
                {
                    //Program.LastErrorMsg = "[ERROR] Run scraper failed,exception=" + e.Message;
                    ret.ErrMsg = "[ERROR] Run scraper failed,exception=" + e.Message;
                    ret.Success = false;
                }
            }

            return ret;
        }

        private static string GetInnerText(IHTMLElement e)
        {
            if (e.innerText == null)
                return "";
            string s = e.innerText;
            if (s.Length > 100)
                s = s.Substring(0, 100);
            return s;
        }
    }

    public class ProgressInfo
    {
        public int JobIndex;
        public int ThreadIndex;
        public int CurrentUrlIndex;
        public int TotalUrlCount;
        public string Url;
        public bool ValidFlag;
        public string ErrMsg;
    }

    public class TResult
    {
        public bool Success;
        public string ErrMsg;
        public object returnvalue;
        public TResult()
        {
            Success = true;
            ErrMsg = string.Empty;
            returnvalue = null;
        }
    }    

    [Serializable]
    public class ScraperRow
    {
        [XmlAttribute(AttributeName = "FieldName")]
        public string FieldName;
        [XmlAttribute(AttributeName = "MakerBefore")]
        public string MakerBefore;
        [XmlAttribute(AttributeName = "MakerAfter")]
        public string MakerAfter;
        [XmlAttribute(AttributeName = "Method")]
        public string Method;
        [XmlAttribute(AttributeName = "Arg")]
        public string Arg;

        public ScraperRow()
        {
            // TODO: Complete member initialization
            this.FieldName = "";
            this.MakerBefore = "";
            this.MakerAfter = "";
            this.Method = "";
            this.Arg = "";
        }

        public ScraperRow(string sFieldName, string sMakerBefore, string sMakerAfter, string sMethod, string sArg)
        {
            // TODO: Complete member initialization
            this.FieldName = sFieldName;
            this.MakerBefore = sMakerBefore;
            this.MakerAfter = sMakerAfter;
            this.Method = sMethod;
            this.Arg = sArg;
        }
    }

    [Serializable]
    [XmlRoot("ScraperData")]
    public class ScraperData
    {
        [XmlAttribute(AttributeName = "ScraperType")]
        public string ScraperType = "XLScraper";
        [XmlAttribute(AttributeName = "TestUrl")]
        public string sTestUrl;
        [XmlElement(ElementName = "ScraperRow")]
        public List<ScraperRow> ScraperRows;
        public ScraperData()
        {
            ScraperRows = new List<ScraperRow> { };
        }
        public void AddScraperRow(string sFieldName, string sMakerBefore, string sMakerAfter, string sMethod, string sArg)
        {
            ScraperRows.Add(new ScraperRow(sFieldName, sMakerBefore, sMakerAfter, sMethod, sArg));
        }
        public void DelScraperRow(string sFieldName)
        {
            for (int i = 0; i < ScraperRows.Count; i++)
                if (ScraperRows[i].FieldName == sFieldName)
                    ScraperRows.RemoveAt(i);
        }
        public void EmptyScraperRows()
        {
            ScraperRows.Clear();
        }
        public static void SaveToFile(string sPath, ScraperData sd)
        {
            if (sd == null || sPath.Length == 0)
            {
                MessageBox.Show("No ScraperData or path is null");
                return;
            }

            FileStream fs = null;

            try
            {
                XmlSerializer xs = new XmlSerializer(typeof(ScraperData));
                fs = new FileStream(sPath, FileMode.Create, FileAccess.Write);
                xs.Serialize(fs, sd);
                fs.Close();
            }
            catch (Exception e)
            {
                if (fs != null)
                    fs.Close();
                MessageBox.Show("Xml serialization failed!\n" + e.Message);
            }
        }
        public static void LoadFromFile(string sPath, ref ScraperData sd)
        {
            FileStream fs = null;
            try
            {
                XmlSerializer xs = new XmlSerializer(typeof(ScraperData));
                fs = new FileStream(sPath, FileMode.Open, FileAccess.Read);
                sd = (ScraperData)xs.Deserialize(fs);
                fs.Close();
            }
            catch
            {
                MessageBox.Show("Xml deserialization failed!");
                if (fs != null)
                    fs.Close();
            }
        }
    }

    [Serializable]
    public class EasyScraper
    {
        [XmlAttribute(AttributeName = "ScraperType")]
        public string ScraperType = "EasyScraper";
        [XmlAttribute(AttributeName = "TestUrl")]
        public string sTestUrl;
        [XmlAttribute(AttributeName = "Level")]
        public int iLevel;
        [XmlAttribute(AttributeName = "ParentTag")]
        public string sParentTag;
        [XmlAttribute(AttributeName = "ChildrenTags")]
        public string sChildrenTags;
        [XmlAttribute(AttributeName = "FieldNames")]
        public string sFieldNames;
        [XmlAttribute(AttributeName = "ValidFlags")]
        public string sValidFlags;

        public EasyScraper()
        {
        }
        public static void SaveToFile(string sPath, EasyScraper es)
        {
            if (es == null || sPath.Length == 0)
            {
                MessageBox.Show("No ScraperData or path is null");
                return;
            }

            FileStream fs = null;

            try
            {
                XmlSerializer xs = new XmlSerializer(typeof(EasyScraper));
                fs = new FileStream(sPath, FileMode.Create, FileAccess.Write);
                xs.Serialize(fs, es);
                fs.Close();
            }
            catch (Exception e)
            {
                if (fs != null)
                    fs.Close();
                MessageBox.Show("Xml serialization failed!\n" + e.Message);
            }

        }
        public static EasyScraper LoadFromFile(string sPath)
        {
            EasyScraper es = new EasyScraper();
            FileStream fs = null;
            try
            {
                XmlSerializer xs = new XmlSerializer(typeof(EasyScraper));
                fs = new FileStream(sPath, FileMode.Open, FileAccess.Read);
                es = (EasyScraper)xs.Deserialize(fs);
                fs.Close();
            }
            catch
            {
                MessageBox.Show("Xml deserialization failed!");
                es = null;
                if (fs != null)
                    fs.Close();
            }
            return es;
        }
    }

    [ComVisible(true),
    ComImport(),
    Guid("7FD52380-4E07-101B-AE2D-08002B2EC713"),
    InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
    interface IPersistStreamInit
    {
        void GetClassID(ref Guid classId);
        [PreserveSig()]
        int IsDirty();
        [PreserveSig()]
        int Load(IStream pstm);
        [PreserveSig()]
        int Save(IStream pstm, bool clearDirty);
        [PreserveSig()]
        int GetMax([In(), Out(), MarshalAs(UnmanagedType.U8)] ref long pcbSize);
        [PreserveSig()]
        int InitNew();
    }
}
